﻿using System;
using System.Collections.Generic;
using System.Xml;
using System.IO;
using ARD.Common;
using ARD.Configuration;
using ARD.EventRouting;
using ARD.ServiceManagement;
using System.Linq;
using SharpObjects.SearchEngine.Common;
using SharpObjects.SearchEngine.Common.ApplicationEvents;
using SharpObjects.SearchEngine.Common.Search;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Store;
using Lucene.Net.Search;

namespace SharpObjects.SearchEngine.Search
{
    public class SearchService : IConfigurationObject, IService, ISearchService
    {
        protected EventBus eventBus = null;
        protected Logging logging = null;
        protected Lucene.Net.Analysis.Analyzer analyzer;
        protected QueryFactory queryFactory;
        protected int maxTermCount;
        protected int desirableTermCount;
        protected double thhresholdFactor;

        #region Constructors

        public SearchService()
        {
            this.analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT);
            this.queryFactory = new QueryFactory() {Analyzer = analyzer};
        }

        #endregion // Constructors

        #region ISearchService Members

        public List<IndexDirectoryDescriptor> IndexDirectories { get; set; }

        #endregion

        #region IService Members

        public string ServiceId
        {
            get { return this.Id; }
        }

        public ServiceManager ServiceManager { get; set; }

        public Type ServiceType
        {
            get { return typeof(IService); }
        }

        #endregion

        #region IConfigurationObject Members

        public string Id { get; set; }

        public void Initialize(XmlElement initialization_section)
        {
            XmlConfig init = new XmlConfig(initialization_section);
            this.Id = init.GetString("@id", this.Id);

            this.IndexDirectories = init.GetObjects<IndexDirectoryDescriptor>("index_directory", IndexDirectoryDescriptor.Assembly, IndexDirectoryDescriptor.Class);

            for (int k = 0; k < this.IndexDirectories.Count; k++)
            {
                IndexDirectoryDescriptor indexDir = this.IndexDirectories[k];

                if (!Path.IsPathRooted(indexDir.Path))
                {
                    DirectoryInfo di = new DirectoryInfo(indexDir.Path);
                    indexDir.Path = di.FullName;
                }
            }

            this.maxTermCount = init.GetInt("max_term_count", 30);
            this.desirableTermCount = init.GetInt("desirable_term_count", 10);
            this.thhresholdFactor = init.GetDouble("characterizer_threshold_factor", 0.5);

            this.eventBus = ServiceManager.Instance.GetService<EventBus>("default_event_bus");
            this.logging = new Logging(this.eventBus);
            this.eventBus.Subscribe(typeof(SearchApplicationEvent), new EventBus.EventHandler(SearchEventHandler));
            this.eventBus.Subscribe(typeof(GetCharacterizerApplicationEvent), new EventBus.EventHandler(GetCharacterizerEventHandler));
        }

        #endregion

        protected void GetCharacterizerEventHandler(IApplicationEvent ev)
        {
            Location location = new Location("SearchService.GetCharacterizerEventHandler()");

            lock (this)
            {
                if (ev is GetCharacterizerApplicationEvent)
                {
                    GetCharacterizerApplicationEvent charEv = (GetCharacterizerApplicationEvent)ev;

                    if (string.IsNullOrEmpty(charEv.IndexDirectoryId))
                    {
                        charEv.IndexDirectoryId = IndexDirectoryDescriptor.DefaultIndexDirectoryId;
                    }

                    string indexDirectory = GetIndexDirectoryPath(charEv.IndexDirectoryId);
                    DirectoryInfo di = new DirectoryInfo(indexDirectory);

                    if (!di.Exists)
                    {
                        string message = string.Format("Index directory does not exst: '{0}'", indexDirectory);
                        this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.ErrorMessage(location) { Message = message }));
                        throw new Exception(message);
                    }

                    Lucene.Net.Store.Directory indexDir = FSDirectory.Open(di);

                    if (IndexReader.IndexExists(indexDir))
                    {
                        IndexReader reader = null;
                        IndexSearcher searcher = null;

                        try
                        {
                            reader = IndexReader.Open(indexDir, false);
                            searcher = new IndexSearcher(reader);

                            TermFreqVector termVector = GetTermVector(reader, searcher, charEv.Uri, charEv.Field);

                            if(termVector == null)
                            {
                                return;
                            }

                            int maxTermCount = this.maxTermCount;

                            if(charEv.MaxTermCount > -1)
                            {
                                maxTermCount = charEv.MaxTermCount;
                            }

                            int desirableTermCount = this.desirableTermCount;

                            if(charEv.DesirableTermCount > -1)
                            {
                                desirableTermCount = charEv.DesirableTermCount;
                            }

                            double thresholdFactor = this.thhresholdFactor;

                            if(charEv.ThresholdFactor >= 0)
                            {
                                thresholdFactor = charEv.ThresholdFactor;
                            }

                            charEv.Characterizer = GetCharacterizer(reader, termVector, maxTermCount, desirableTermCount, thhresholdFactor);
                        }
                        catch (Exception ex)
                        {
                            this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.ErrorMessage(location, ex)));
                            throw;
                        }
                        finally
                        {
                            if (reader != null)
                            {
                                reader.Close();
                            }

                            if (searcher != null)
                            {
                                searcher.Close();
                            }
                        }
                    }
                }
            }
        }

        private TermFreqVector GetTermVector(IndexReader reader, IndexSearcher searcher, string uri, string fieldName)
        {
            IQueryConstraint uriConstraint =
                new SharpObjects.SearchEngine.Common.QueryConstraints.TermQueryConstraint() { FieldName = DefaultIndexFields.uri_field, TermText = uri };

            Query query = this.queryFactory.CreateQuery(uriConstraint);
            TopDocs topDocs = searcher.Search(query, 1);

            if(topDocs.scoreDocs.Length == 0)
            {
                return null;
                //throw new ArgumentException("URI not found in the index: " + uri);
            }

            int documentId = topDocs.scoreDocs[0].doc;

            TermFreqVector termVector = reader.GetTermFreqVector(documentId, fieldName);

            return termVector;
        }

        private static Dictionary<string, double> GetCharacterizer(IndexReader reader, TermFreqVector termVector, int maxTermCount, int desirableTermCount, double thresholdFactor)
        {
            Dictionary<string, double> characterizer = new Dictionary<string, double>();

            double minTfIdf;
            double maxTfIdf;
            List<TermVectorItem> tfidfs = GetTfIdfs(reader, termVector, out maxTfIdf, out minTfIdf);

            tfidfs.Sort(new TermVectorItemComparer(isScoreNormalized: true));

            double threshold = minTfIdf + (maxTfIdf - minTfIdf) * thresholdFactor;

            int termCount = 0;

            for (int k = 0; k < Math.Min(maxTermCount, tfidfs.Count); k++)
            {
                if (termCount < desirableTermCount
                    || tfidfs[k].Score >= threshold)
                {
                    characterizer.Add(tfidfs[k].Term, tfidfs[k].Score);
                    termCount++;
                }
            }

            return characterizer;
        }

        private static List<TermVectorItem> GetTfIdfs(IndexReader reader, TermFreqVector termVector, out double maxTfIdf, out double minTfIdf)
        {
            maxTfIdf = 0.0;
            minTfIdf = 1.0;

            List<TermVectorItem> tfidfs = new List<TermVectorItem>();

            string[] terms = termVector.GetTerms();
            int[] freqs = termVector.GetTermFrequencies();

            long totalUniqueTermCount = GetUniqueTermCount(reader);
            int totalNumberOfDocs = reader.NumDocs();

            for (int k = 0; k < terms.Length; k++)
            {
                string term = terms[k];
                // TF = term count in the doc / total terms count in the doc
                double TF = ((double)freqs[k]) / ((double)freqs.Length);

                int docFreq = reader.DocFreq(new Term(termVector.GetField(), term));
                int docCount = reader.NumDocs();

                double IDF = ((double)totalNumberOfDocs) / ((double)docFreq);
                double TF_IDF = TF * IDF / ((double)totalNumberOfDocs); // range 0 to 1

                maxTfIdf = Math.Max(maxTfIdf, TF_IDF);
                minTfIdf = Math.Min(minTfIdf, TF_IDF);

                TermVectorItem tfidf = new TermVectorItem() { Score = TF_IDF, Term = term };
                tfidfs.Add(tfidf);
            }

            return tfidfs;
        }

        private static long GetUniqueTermCount(IndexReader reader)
        {
            IndexReader[] readers = reader.GetSequentialSubReaders();
            long count = 0;

            for (int k = 0; k < readers.Length; k++)
            {
                count += readers[k].GetUniqueTermCount();
            }

            return count;
        }

        protected void SearchEventHandler(IApplicationEvent ev)
        {
            Location location = new Location("SearchService.SearchEventHandler()");

            lock (this)
            {
                if (ev is SearchApplicationEvent)
                {
                    SearchApplicationEvent searchEv = (SearchApplicationEvent) ev;

                    if(string.IsNullOrEmpty(searchEv.IndexDirectoryId))
                    {
                        searchEv.IndexDirectoryId = IndexDirectoryDescriptor.DefaultIndexDirectoryId;
                    }

                    string indexDirectory = GetIndexDirectoryPath(searchEv.IndexDirectoryId);
                    DirectoryInfo di = new DirectoryInfo(indexDirectory);

                    if (!di.Exists)
                    {
                        this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.ErrorMessage(location) { Message = string.Format("Index directory does not exst: '{0}'", indexDirectory) }));

                        SearchResultApplicationEvent searchResultApplicationEvent = CreateSearchResultEventObject(searchEv);
                        this.eventBus.Publish(typeof(SearchResultApplicationEvent), searchResultApplicationEvent);
                        
                        return;
                    }

                    Lucene.Net.Store.Directory indexDir = FSDirectory.Open(di);

                    if (IndexReader.IndexExists(indexDir))
                    {
                        IndexReader reader = null;
                        IndexSearcher searcher = null;

                        try
                        {
                            reader = IndexReader.Open(indexDir, false);
                            searcher = new IndexSearcher(reader);

                            Query query = this.queryFactory.CreateQuery(searchEv.Constraint);

                            int max = (searchEv.PageNumber + 1) * searchEv.PageSize;
                            TopDocs topDocs = searcher.Search(query, max);

                            int size = max;

                            if (max > topDocs.scoreDocs.Length)
                            {
                                size = topDocs.scoreDocs.Length;
                            }

                            double maxScore = topDocs.GetMaxScore();

                            SearchResultApplicationEvent searchResultApplicationEvent = CreateSearchResultEventObject(searchEv);

                            int start = searchEv.PageNumber * searchEv.PageSize;

                            for (int k = start; k < size; k++)
                            {
                                IQueryResultItem item = CreateQueryResultItem(searcher, reader, topDocs, k, searchEv, maxScore);
                                searchResultApplicationEvent.QueryResult.QueryResultItems.Add(item);
                            }

                            CalculateConcepts(ref searchResultApplicationEvent);

                            searchEv.QueryResult = searchResultApplicationEvent.QueryResult;

                            this.eventBus.Publish(typeof(SearchResultApplicationEvent), searchResultApplicationEvent);
                        }
                        catch(Exception ex)
                        {
                            this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.ErrorMessage(location, ex)));

                            SearchResultApplicationEvent searchResultApplicationEvent = CreateSearchResultEventObject(searchEv);
                            this.eventBus.Publish(typeof(SearchResultApplicationEvent), searchResultApplicationEvent);

                            this.eventBus.Publish(typeof(SearchResultApplicationEvent), searchResultApplicationEvent);
                            throw;
                        }
                        finally
                        {
                            if (reader != null)
                            {
                                reader.Close();
                            }

                            if (searcher != null)
                            {
                                searcher.Close();
                            }
                        }
                    }
                }
            }
        }

        protected SearchResultApplicationEvent CreateSearchResultEventObject(SearchApplicationEvent searchEv)
        {
            SearchResultApplicationEvent searchResultApplicationEvent = new SearchResultApplicationEvent() { Constraint = searchEv.Constraint };
            searchResultApplicationEvent.QueryResult = new QueryResult() { PageNumber = searchEv.PageNumber, PageSize = searchEv.PageSize };
            return searchResultApplicationEvent;
        }

        protected IQueryResultItem CreateQueryResultItem(IndexSearcher searcher, IndexReader reader, TopDocs topDocs, int ind, SearchApplicationEvent searchEv, double maxScore)
        {
            IQueryResultItem item = new QueryResultItem();

            item.Score = topDocs.scoreDocs[ind].score / maxScore;
            item.DocumentId = topDocs.scoreDocs[ind].doc;
            Document resDoc = searcher.Doc(item.DocumentId);
            PopulateResultItem(searcher, reader, topDocs, item, resDoc);
            
            return item;
        }

        private static void PopulateResultItem(IndexSearcher searcher, IndexReader reader, TopDocs topDocs, IQueryResultItem item, Document resDoc)
        {
            // process default fields - they are always present 
            Field uriField = resDoc.GetField(DefaultIndexFields.uri_field);
            item.Uri = uriField.StringValue();

            Field dateIndexedField = resDoc.GetField(DefaultIndexFields.date_indexed_field);
            item.DateIndexed = Convert.ToDateTime(dateIndexedField.StringValue());

            Field dateModifiedField = resDoc.GetField(DefaultIndexFields.date_modified_field);
            item.DateModified = Convert.ToDateTime(dateModifiedField.StringValue());

            
            // process other fields
            System.Collections.IList fields = resDoc.GetFields();

            foreach (Field field in fields)
            {
                if(!IsDefaultField(field)) // exclude fields already processed above
                {
                    if(field.IsBinary())
                    {
                        //TODO: decide what to do with binary fields                    
                    }
                    else
                    {
                        if (field.IsStored() || field.IsTermVectorStored())
                        {
                            IQueryResultItemField resultItemField;
                            bool isNewField = false;

                            if (item.Fields.ContainsKey(field.Name()))
                            {
                                resultItemField = item.Fields[field.Name()];
                            }
                            else
                            {
                                resultItemField = new QueryResultItemField();
                                item.Fields.Add(field.Name(), resultItemField);
                                isNewField = true;
                            }

                            if (field.IsStored())
                            {
                                resultItemField.Values.Add(field.StringValue());
                            }

                            if (field.IsTermVectorStored() && isNewField)
                            {
                                TermFreqVector termFreqVector = reader.GetTermFreqVector(item.DocumentId, field.Name());
                                string[] terms = termFreqVector.GetTerms();
                                int[] freqs = termFreqVector.GetTermFrequencies();

                                int count = Math.Min(terms.Length, freqs.Length);

                                for(int k = 0; k < count; k++)
                                {
                                    ITermVectorItem termVectorItem = new TermVectorItem
                                                                         {
                                                                             Term = terms[k], 
                                                                             Score = freqs[k]
                                                                         };
                                    resultItemField.TermVector.Add(termVectorItem);
                                }

                                resultItemField.TermVector.Sort(new TermVectorItemComparer());
                            }
                        }
                    }
                }
            }

        }

        private static bool IsDefaultField(Field field)
        {
            switch (field.Name())
            {
                case DefaultIndexFields.uri_field:
                    return true;
                case DefaultIndexFields.date_indexed_field:
                    return true;
                case DefaultIndexFields.date_modified_field:
                    return true;
                default:
                    return false;
            }
        }


        protected void CalculateConcepts(ref SearchResultApplicationEvent searchResultApplicationEvent)
        {
            //TODO: provide implementation

        }

        protected string GetIndexDirectoryPath(string indexDirectoryId)
        {
            var dirPath = (from dir in this.IndexDirectories
                           where dir.Id == indexDirectoryId
                           select dir.Path).Single();

            if(dirPath == null)
            {
                throw new ApplicationException("Index directory not found. ID = " + indexDirectoryId);
            }

            return dirPath;
        }

    }
}
